/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package net.cjb.malacma.rssfeed.crawler;

import com.sun.syndication.feed.synd.SyndFeed;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import net.cjb.malacma.rssfeed.util.FeedFacade;
import net.cjb.malacma.rssfeed.util.FeedFactory;
import websphinx.Crawler;
import websphinx.Page;

import static java.lang.System.*;

public abstract class BaseCrawler extends Crawler {
    private static SyndFeed sf;
    private static Pattern p1 = Pattern.compile("rss", Pattern.CASE_INSENSITIVE);
    private static Pattern p2 = Pattern.compile("xml", Pattern.CASE_INSENSITIVE);


    public static Integer cont = new Integer(0);
    BaseCrawler() {
        super();  // Do what the parent  crawler would do
    }


    // What to do when we visit the page
    @Override
    public void visit(Page page) {
        //System.out.println("Visiting: " + page.getTitle());

        String content = page.getContent();

        // pesquisa ocorrências de RSS
        Matcher m1 = p1.matcher(content);
        Matcher m2 = p2.matcher(content);
        if (m1.find()||m2.find()) {

            try {
                //tenta criar um feed para ver se a notícia é válida
                sf = FeedFactory.getInstance(page.getURL().toString());

                //Grava o feed
                FeedFacade.persistFeedUrl(page.getURL().toString(), sf.getTitle());

                sf = null;
                cont++;
                out.println("------------");
                out.println("FEED TOTAL"+cont);
            } catch (Exception ex) {
                System.out.println("Page does not represent a valid RSS FEED.\nThe following error occured:\n"+ex.toString());
            } finally {
                content = null;
                m1 = null;
                m2 = null;
                // Since we don't need to retain the pages
                // This code helps with memory management
                page.getOrigin().setPage(null);
                page.discardContent();
                page = null;
            }
        }
    }
}